package com.terren.spider.core.html.scheme.impl;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import com.terren.spider.entity.biz.ArticleSearch;
import com.terren.spider.entity.core.Entry;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

public class GoogleNewsScheme extends BasicCustomScheme{

	@Override
	public List<ArticleSearch> findTitleInfo(Page page, Entry entry) {
		List<ArticleSearch> articleList = new ArrayList<ArticleSearch>();
		//----------------------------------爬取搜索页面的标题，发布时间，url
		Html pageHtml = page.getHtml();
		List<String> partList = pageHtml.xpath("//div[@class='g']").all();//待爬取的新闻列表
		for(int i=1;i<=partList.size();i++){
			ArticleSearch temp = new ArticleSearch();
			temp.setTitle(pageHtml.xpath("//*[@id='rso']/div[1]/div["+i+"]/div/div[1]/h3/a/text()").get());
			temp.setNewsSource(pageHtml.xpath("//*[@id='rso']/div[1]/div["+i+"]/div/div[1]/div[1]/span[1]/text()").get());
			temp.setPubTimeStr(pageHtml.xpath("//*[@id='rso']/div[1]/div["+i+"]/div/div[1]/div[1]/span[3]/text()").get());
			String url = pageHtml.xpath("//*[@id='rso']/div[1]/div["+i+"]/div/div[1]/h3/a").links().get();
			temp.setUrl(url);
			temp.setSourceId(entry.getSourceId());
			temp.setTaskId(entry.getTaskId());
			//url不为空才能添加
			if(!"".equals(url)&&url!=null){
				articleList.add(temp);
			}
		}
		return articleList;
	}

	@Override
	public Set<String> findHelpPaginationUrl(Page page, Entry entry) {
		List<String> pageUrls = page.getHtml().$("#navcnt").links().all();
		String nextUrl = page.getHtml().$("#pnnext").links().get();//获取下一页链接
		Set<String> resultSet = new HashSet<>();
		resultSet.addAll(pageUrls);
		resultSet.add(nextUrl);
		return resultSet;
	}
}
